{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from h1st.model.predictive_model import PredictiveModel\n",
    "class RuleModel(PredictiveModel):\n",
    "    # 98 percentile\n",
    "    daily_thresholds = {\n",
    "        \"volt\": 188.83,  # >\n",
    "        \"rotate\": 373.05,  # <\n",
    "        \"vibration\": 49.38,  # >\n",
    "    }\n",
    "\n",
    "    def predict_pointwise(self, input_data):\n",
    "        daily_average = np.average(\n",
    "            input_data.reshape((-1, 4)), axis=0\n",
    "        )\n",
    "        volt_val = daily_average[0]\n",
    "        rotate_val = daily_average[1]\n",
    "        vibration_val = daily_average[3]\n",
    "\n",
    "        pred = {\"comp1\": 0, \"comp2\": 0, \"comp4\": 0}\n",
    "        pred['comp1'] = 1 if volt_val > self.daily_thresholds[\"volt\"] else 0\n",
    "        pred['comp2'] = 1 if rotate_val < self.daily_thresholds[\"rotate\"] else 0\n",
    "        pred['comp4'] = 1 if vibration_val > self.daily_thresholds[\"vibration\"] else 0\n",
    "        return pred\n",
    "\n",
    "    def predict(self, input_data):\n",
    "        df = input_data['X']\n",
    "        return {'predictions': pd.DataFrame(\n",
    "            map(self.predict_pointwise, df.values), \n",
    "        )}  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from h1st.model.fuzzy import (\n",
    "    FuzzyVariables,\n",
    "    FuzzyMembership as fm,\n",
    "    FuzzyRules,\n",
    "    FuzzyModel,\n",
    "    FuzzyModeler\n",
    ")\n",
    "\n",
    "def get_metadata(data):\n",
    "    res = {}\n",
    "    for k, v in dict(data.describe().loc['max']).items():\n",
    "        res[k] = {'max': v}\n",
    "    for k, v in dict(data.describe().loc['min']).items():\n",
    "        res[k].update({'min': v})    \n",
    "    return res\n",
    "\n",
    "def create_fuzzy_model(data):\n",
    "    metadata = get_metadata(data) # df_model3_daily\n",
    "    fuzzy_vars = FuzzyVariables()\n",
    "    fuzzy_vars.add(\n",
    "        var_name='volt',\n",
    "        var_type='antecedent',\n",
    "        var_range=np.arange(\n",
    "            metadata['volt']['min'], \n",
    "            metadata['volt']['max'], \n",
    "            0.1\n",
    "        ),\n",
    "        membership_funcs=[('high', fm.SIGMOID, [188.83, 0.25]), # 180\n",
    "                        ('low', fm.SIGMOID, [188.83, -0.25])]\n",
    "    )\n",
    "    fuzzy_vars.add(\n",
    "        var_name='rotate',\n",
    "        var_type='antecedent',\n",
    "        var_range=np.arange(\n",
    "            metadata['rotate']['min'], \n",
    "            metadata['rotate']['max'], \n",
    "            0.1\n",
    "        ),\n",
    "        membership_funcs=[('high', fm.SIGMOID, [373.05, 0.06]), # 400\n",
    "                        ('low', fm.SIGMOID, [373.05, -0.15])]\n",
    "    )\n",
    "    fuzzy_vars.add(\n",
    "        var_name='vibration',\n",
    "        var_type='antecedent',\n",
    "        var_range=np.arange(\n",
    "            metadata['vibration']['min'], \n",
    "            metadata['vibration']['max'], \n",
    "            0.1\n",
    "        ),\n",
    "        membership_funcs=[('high', fm.SIGMOID, [49.38, 0.5]), # 44 ?\n",
    "                        ('low', fm.SIGMOID, [49.38, -0.5])]\n",
    "    )\n",
    "    fuzzy_vars.add(\n",
    "        var_name='comp1',\n",
    "        var_type='consequent',\n",
    "        var_range=np.arange(0, 1+1e-5, 0.1),\n",
    "        membership_funcs=[('false', fm.GAUSSIAN, [0, 0.4]),\n",
    "                        ('true', fm.GAUSSIAN, [1, 0.4])]\n",
    "    )\n",
    "    fuzzy_vars.add(\n",
    "        var_name='comp2',\n",
    "        var_type='consequent',\n",
    "        var_range=np.arange(0, 1+1e-5, 0.1),\n",
    "        membership_funcs=[('false', fm.GAUSSIAN, [0, 0.4]),\n",
    "                        ('true', fm.GAUSSIAN, [1, 0.4])]\n",
    "    )\n",
    "    fuzzy_vars.add(\n",
    "        var_name='comp4',\n",
    "        var_type='consequent',\n",
    "        var_range=np.arange(0, 1+1e-5, 0.1),\n",
    "        membership_funcs=[('false', fm.GAUSSIAN, [0, 0.4]),\n",
    "                        ('true', fm.GAUSSIAN, [1, 0.4])]\n",
    "    )\n",
    "\n",
    "    fuzzy_rule = FuzzyRules()\n",
    "    fuzzy_rule.add(\n",
    "        'rule1',\n",
    "        if_term=fuzzy_vars.get('volt')['high']&fuzzy_vars.get('rotate')['high']&fuzzy_vars.get('vibration')['low'],\n",
    "        then_term=fuzzy_vars.get('comp1')['true']\n",
    "    )\n",
    "    fuzzy_rule.add(\n",
    "        'rule2',\n",
    "        if_term=fuzzy_vars.get('rotate')['low']&fuzzy_vars.get('volt')['low']&fuzzy_vars.get('vibration')['low'],\n",
    "        then_term=fuzzy_vars.get('comp2')['true']\n",
    "    )\n",
    "    fuzzy_rule.add(\n",
    "        'rule3',\n",
    "        if_term=fuzzy_vars.get('vibration')['high']&fuzzy_vars.get('volt')['low']&fuzzy_vars.get('rotate')['high'],\n",
    "        then_term=fuzzy_vars.get('comp4')['true']\n",
    "    )\n",
    "    fuzzy_rule.add(\n",
    "        'rule4',\n",
    "        if_term=fuzzy_vars.get('volt')['low'],\n",
    "        then_term=fuzzy_vars.get('comp1')['false']\n",
    "    )\n",
    "    fuzzy_rule.add(\n",
    "        'rule5',\n",
    "        if_term=fuzzy_vars.get('rotate')['high'],\n",
    "        then_term=fuzzy_vars.get('comp2')['false']\n",
    "    )\n",
    "    fuzzy_rule.add(\n",
    "        'rule6',\n",
    "        if_term=fuzzy_vars.get('vibration')['low'],\n",
    "        then_term=fuzzy_vars.get('comp4')['false']\n",
    "    )\n",
    "\n",
    "    class CustomFuzzyModel(FuzzyModel):\n",
    "        def process_rule(self, input_data: dict) -> dict:\n",
    "            if self.rule_engine is None:\n",
    "                raise ValueError(\n",
    "                    (\n",
    "                        \"Property rule_engine is None. Please load your rule_engine \"\n",
    "                        \"to run this method.\"\n",
    "                    )\n",
    "                )\n",
    "\n",
    "            input_data = np.array(list(input_data.values()))\n",
    "            daily_average = np.average(\n",
    "                input_data.reshape((-1, 4)), axis=0\n",
    "            )\n",
    "            input_data = {\n",
    "                \"volt\": daily_average[0],\n",
    "                \"rotate\": daily_average[1],\n",
    "                \"vibration\": daily_average[3]\n",
    "            } \n",
    "            for key, value in input_data.items():\n",
    "                self.rule_engine.input[key] = value\n",
    "            self.rule_engine.compute()\n",
    "\n",
    "            outputs = {}\n",
    "            for cls in self.rule_engine.ctrl.consequents:\n",
    "                outputs[cls.label] = round(self.rule_engine.output[cls.label], 5)\n",
    "            return outputs\n",
    "\n",
    "    modeler = FuzzyModeler(model_class=CustomFuzzyModel)\n",
    "    teacher = modeler.build_model(fuzzy_vars, fuzzy_rule)\n",
    "    return teacher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model1 failure records: (189, 3)\n",
      "model2 failure records: (168, 3)\n",
      "model3 failure records: (221, 3)\n",
      "model4 failure records: (183, 3)\n",
      "volt: 178.994274651537\n",
      "rotate: 405.58135851278706\n",
      "vibration: 43.93598497211916\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# you can download the data from kaggle using the following link.\n",
    "# https://www.kaggle.com/datasets/arnabbiswas1/microsoft-azure-predictive-maintenance\n",
    "\n",
    "import os\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "dir_path = os.getcwd() + '/azure_iot'\n",
    "path = f'{dir_path}/'\n",
    "telemetry_url = 'PdM_telemetry.csv'\n",
    "df = pd.read_csv(path + 'PdM_telemetry.csv')\n",
    "df.loc[:, 'datetime'] = pd.to_datetime(df['datetime'])\n",
    "df.loc[:, 'datetime'] = df['datetime'] - pd.Timedelta(hours=6)\n",
    "df_machines = pd.read_csv(path + 'PdM_machines.csv')\n",
    "df = df.join(df_machines.set_index('machineID'), on='machineID')\n",
    "df.loc[:, 'date'] = df['datetime'].dt.date\n",
    "df_failures = pd.read_csv(path + 'PdM_failures.csv')\n",
    "df_failures.shape\n",
    "\n",
    "for model_type in ['model1', 'model2', 'model3', 'model4']:\n",
    "    id_list = df[df.model == model_type].machineID.unique()\n",
    "    print(f'{model_type} failure records:', df_failures[df_failures.machineID.isin(id_list)].shape)\n",
    "\n",
    "df_model3 = df[df.model=='model3']\n",
    "df_model3 = df_model3[df_model3.date != datetime(2016, 1, 1).date()]\n",
    "df_model3.shape\n",
    "\n",
    "df_model3_failures = df_failures[df_failures.machineID.isin(df_model3.machineID.unique())]\n",
    "df_model3_failures.shape\n",
    "\n",
    "df_model3_failures['datetime'] = pd.to_datetime(df_model3_failures['datetime'])\n",
    "df_model3_failures['date'] = df_model3_failures['datetime'].apply(lambda x: x.date())\n",
    "df_model3_failures['date_1'] = df_model3_failures['date'] - timedelta(days=1)\n",
    "df_model3_failures.failure.value_counts()\n",
    "\n",
    "df_model3_daily = df_model3.groupby(['date', 'machineID']).agg('mean')\n",
    "\n",
    "percentile = 0.96\n",
    "print('volt:', df_model3_daily['volt'].quantile(percentile))\n",
    "print('rotate:', df_model3_daily['rotate'].quantile(1-percentile))\n",
    "print('vibration:', df_model3_daily['vibration'].quantile(percentile))\n",
    "\n",
    "df_model3.machineID.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['volt_0', 'rotate_0', 'pressure_0', 'vibration_0', 'volt_1', 'rotate_1', 'pressure_1', 'vibration_1', 'volt_2', 'rotate_2', 'pressure_2', 'vibration_2', 'volt_3', 'rotate_3', 'pressure_3', 'vibration_3', 'volt_4', 'rotate_4', 'pressure_4', 'vibration_4', 'volt_5', 'rotate_5', 'pressure_5', 'vibration_5', 'volt_6', 'rotate_6', 'pressure_6', 'vibration_6', 'volt_7', 'rotate_7', 'pressure_7', 'vibration_7', 'volt_8', 'rotate_8', 'pressure_8', 'vibration_8', 'volt_9', 'rotate_9', 'pressure_9', 'vibration_9', 'volt_10', 'rotate_10', 'pressure_10', 'vibration_10', 'volt_11', 'rotate_11', 'pressure_11', 'vibration_11', 'volt_12', 'rotate_12', 'pressure_12', 'vibration_12', 'volt_13', 'rotate_13', 'pressure_13', 'vibration_13', 'volt_14', 'rotate_14', 'pressure_14', 'vibration_14', 'volt_15', 'rotate_15', 'pressure_15', 'vibration_15', 'volt_16', 'rotate_16', 'pressure_16', 'vibration_16', 'volt_17', 'rotate_17', 'pressure_17', 'vibration_17', 'volt_18', 'rotate_18', 'pressure_18', 'vibration_18', 'volt_19', 'rotate_19', 'pressure_19', 'vibration_19', 'volt_20', 'rotate_20', 'pressure_20', 'vibration_20', 'volt_21', 'rotate_21', 'pressure_21', 'vibration_21', 'volt_22', 'rotate_22', 'pressure_22', 'vibration_22', 'volt_23', 'rotate_23', 'pressure_23', 'vibration_23']\n"
     ]
    }
   ],
   "source": [
    "keys = ['machineID', 'date']\n",
    "features = ['volt', 'rotate', 'pressure', 'vibration']\n",
    "column_values = []\n",
    "\n",
    "for idx in range(24):\n",
    "    column_values.extend([f'{feature}_{idx}' for feature in features])\n",
    "print(column_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(list_of_daily_data, features):\n",
    "    x_train_list = []\n",
    "    y_train_list = []\n",
    "    for idx, df_daily_one in list_of_daily_data:\n",
    "        mid = idx[0]\n",
    "        date = idx[1]\n",
    "\n",
    "        if df_daily_one.shape[0] != 24:\n",
    "            continue\n",
    "\n",
    "        df_filtered_f = df_model3_failures[\n",
    "            (df_model3_failures.date_1==date)&(df_model3_failures.machineID==mid)]\n",
    "        y_label = {\"comp1\": 0, \"comp2\": 0, \"comp4\": 0}\n",
    "        if df_filtered_f.shape[0] >= 1:\n",
    "            for i in range(df_filtered_f.shape[0]):\n",
    "                y_label.update({df_filtered_f['failure'].iloc[i]: 1})\n",
    "        x_train_list.append(np.concatenate(df_daily_one[features].values).tolist())\n",
    "        y_train_list.append(y_label)\n",
    "    return x_train_list, y_train_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import resample\n",
    "\n",
    "def handle_data_imbalance(df_x, df_y):\n",
    "    df_train_y_normal = df_y[df_y.sum(axis=1) == 0]\n",
    "    df_train_y_normal_resampled = resample(df_train_y_normal,\n",
    "                replace=True,\n",
    "                n_samples=100,\n",
    "                random_state=42)\n",
    "    df_train_x_normal_resampled = df_x.filter(items = df_train_y_normal_resampled.index, axis=0)\n",
    "    assert all(df_train_y_normal_resampled.head().index == df_train_x_normal_resampled.head().index)\n",
    "\n",
    "    df_train_y_abnormal = df_y[df_y.sum(axis=1) != 0]\n",
    "    df_train_y_abnormal_resampled = resample(df_train_y_abnormal,\n",
    "                replace=True,\n",
    "                n_samples=600,\n",
    "                random_state=42)\n",
    "    df_train_y_abnormal_resampled.shape\n",
    "    df_train_x_abnormal_resampled = df_x.filter(items = df_train_y_abnormal_resampled.index, axis=0)\n",
    "    df_train_x_abnormal_resampled.shape\n",
    "    assert all(df_train_y_abnormal_resampled.head().index == df_train_x_abnormal_resampled.head().index)\n",
    "\n",
    "    df_train_y_final = pd.concat([df_train_y_normal_resampled, df_train_y_abnormal_resampled], axis=0)\n",
    "    df_train_x_final = pd.concat([df_train_x_normal_resampled, df_train_x_abnormal_resampled], axis=0)\n",
    "\n",
    "    return df_train_x_final, df_train_y_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-10 10:02:57.929 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:03:04.970 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:03:08.157 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:03:18.291 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:03:20.711 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:03:26.648 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:03:37.210 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:03:45.947 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:03:49.171 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:04:00.472 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:04:03.090 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:04:10.790 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "2022-10-10 10:04:22.094 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:04:29.506 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:04:32.559 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:04:41.609 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "2022-10-10 10:04:43.929 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:04:49.712 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:05:00.001 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:05:08.751 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:05:14.303 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:05:26.164 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:05:28.823 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:05:35.475 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:05:44.845 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:05:52.018 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:05:54.813 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "X has feature names, but StandardScaler was fitted without feature names\n",
      "2022-10-10 10:06:03.145 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:06:05.374 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n",
      "2022-10-10 10:06:11.137 | INFO     | h1st.model.oracle.oracle_modeler:build_model:142 - Evaluated all sub models successfully.\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "from sklearn.model_selection import KFold\n",
    "from h1st.model.oracle import OracleModeler\n",
    "from h1st.model.oracle.ensembler_modelers import MLPEnsembleModeler\n",
    "\n",
    "# prepare cross validation\n",
    "kfold = KFold(n_splits=5, shuffle=True, random_state=3)\n",
    "\n",
    "all_metrics = defaultdict(lambda: defaultdict(list))\n",
    "all_index = defaultdict(lambda: defaultdict(list))\n",
    "\n",
    "# 1. build rule-based model\n",
    "bool_teacher = RuleModel()\n",
    "fuzzy_teacher = create_fuzzy_model(df_model3_daily)\n",
    "fuzzy_thresholds = {'comp1': 0.46, 'comp2': 0.48, 'comp4': 0.46}\n",
    "\n",
    "for no, m_ids in enumerate(kfold.split(df_model3.machineID.unique())):\n",
    "    train_id, test_id = m_ids\n",
    "    df_train = df_model3[df_model3.machineID.isin(train_id)]\n",
    "    df_test = df_model3[df_model3.machineID.isin(test_id)]\n",
    "\n",
    "    temp_gb = df_train.groupby(keys)\n",
    "    list_of_train_daily = [item for item in temp_gb]\n",
    "\n",
    "    temp_gb = df_test.groupby(keys)\n",
    "    list_of_test_daily = [item for item in temp_gb]\n",
    "\n",
    "    # print(len(train_id), len(test_id),df_train.shape, df_test.shape)\n",
    "    # print(f'number of data points in train dataset: {len(list_of_train_daily)}')\n",
    "    # print(f'number of data points in test dataset: {len(list_of_test_daily)}')\n",
    "\n",
    "    \n",
    "    x_train_list, y_train_list = preprocess_data(list_of_train_daily, features)\n",
    "    x_test_list, y_test_list = preprocess_data(list_of_test_daily, features)\n",
    "\n",
    "    df_train_x = pd.DataFrame(x_train_list, columns=column_values)\n",
    "    df_train_y = pd.DataFrame(y_train_list)\n",
    "    df_test_x = pd.DataFrame(x_test_list, columns=column_values)\n",
    "    df_test_y = pd.DataFrame(y_test_list)\n",
    "\n",
    "    df_train_x_final, df_train_y_final = handle_data_imbalance(df_train_x, df_train_y)\n",
    "    # print(df_train_x.shape, df_train_y.shape, df_test_x.shape, df_test_y.shape)\n",
    "    # print(no, df_train_x_final.shape, df_train_y_final.shape)   \n",
    "    input_data = {\n",
    "        \"unlabeled_data\": df_train_x_final,\n",
    "        \"labeled_data\": {\n",
    "            \"X_train\": df_train_x_final.reset_index(drop=True),\n",
    "            \"y_train\": df_train_y_final.reset_index(drop=True),\n",
    "            \"X_test\": df_test_x,\n",
    "            \"y_test\": df_test_y,\n",
    "        },\n",
    "    }\n",
    "\n",
    "    # 3. build oracle\n",
    "    # 3.1 bool\n",
    "    modeler = OracleModeler()\n",
    "    oracle_with_bool = modeler.build_model(\n",
    "        data=input_data,\n",
    "        teacher_model=bool_teacher)\n",
    "\n",
    "    # # 3.2 fuzzy\n",
    "    oracle_with_fuzzy = modeler.build_model(\n",
    "        data=input_data,\n",
    "        teacher_model=fuzzy_teacher,\n",
    "        fuzzy_thresholds=fuzzy_thresholds)\n",
    "\n",
    "    # 3.3 bool + ml_ensemble\n",
    "    oracle_with_bool_ml = modeler.build_model(\n",
    "        data=input_data, \n",
    "        teacher_model=bool_teacher,\n",
    "        ensembler_modeler=MLPEnsembleModeler)\n",
    "\n",
    "    # 3.4 fuzzy + ml_ensemble\n",
    "    oracle_with_fuzzy_ml = modeler.build_model(\n",
    "        data=input_data, \n",
    "        teacher_model=fuzzy_teacher,\n",
    "        fuzzy_thresholds=fuzzy_thresholds,\n",
    "        ensembler_modeler=MLPEnsembleModeler) \n",
    "\n",
    "    # 3.5 bool + ml_ensemble + x\n",
    "    oracle_with_bool_ml_x = modeler.build_model(\n",
    "        data=input_data, \n",
    "        teacher_model=bool_teacher,\n",
    "        ensembler_modeler=MLPEnsembleModeler,\n",
    "        inject_x_in_ensembler=True)\n",
    "\n",
    "    # 3.6 fuzzy + ml_ensemble + x\n",
    "    oracle_with_fuzzy_ml_x = modeler.build_model(\n",
    "        data=input_data, \n",
    "        teacher_model=fuzzy_teacher,\n",
    "        fuzzy_thresholds=fuzzy_thresholds,\n",
    "        ensembler_modeler=MLPEnsembleModeler,\n",
    "        inject_x_in_ensembler=True) \n",
    "\n",
    "    model_map = {\n",
    "        \"oracle_with_bool\": oracle_with_bool,\n",
    "        \"oracle_with_fuzzy\": oracle_with_fuzzy,\n",
    "        \"oracle_with_bool_ml\": oracle_with_bool_ml,\n",
    "        \"oracle_with_fuzzy_ml\": oracle_with_fuzzy_ml,\n",
    "        \"oracle_with_bool_ml_x\": oracle_with_bool_ml_x,\n",
    "        \"oracle_with_fuzzy_ml_x\": oracle_with_fuzzy_ml_x,        \n",
    "    }\n",
    "\n",
    "    # 4. collect evaluation results\n",
    "    for name, oracle in model_map.items():\n",
    "        for metrics in [\"f1_score\", 'precision', 'recall']:\n",
    "            for label in ['comp1', 'comp2', 'comp4']:\n",
    "                temp = oracle.metrics[metrics][label]\n",
    "                s1, s2 = temp.pop('students')\n",
    "                temp.update({'student1': s1, 'student2': s2})\n",
    "                all_metrics[metrics][name].append({label: temp})\n",
    "                # all_index[metrics][name].append(f'{label}_{name}_{no}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_avg_metrics = []\n",
    "oracle_names = []\n",
    "for metrics_type in ['f1_score', 'precision', 'recall']:\n",
    "    for oracle_name, metrics in all_metrics[metrics_type].items():\n",
    "\n",
    "        temp = defaultdict(list)\n",
    "        for m in metrics:\n",
    "            for label, met in m.items():\n",
    "                temp[label].append(met)\n",
    "\n",
    "        for label in temp:\n",
    "            df_metrics = pd.DataFrame(temp[label])\n",
    "            # print(metrics_type, oracle_name, df_metrics.mean().values)\n",
    "            all_avg_metrics.append(\n",
    "                [metrics_type, label] + list(df_metrics.mean().values)\n",
    "            )\n",
    "            oracle_names.append(f'{label}_{oracle_name}')\n",
    "\n",
    "final_metrics = pd.DataFrame(\n",
    "    all_avg_metrics,\n",
    "    columns=['metrics_type', 'label', 'teacher', 'ensemblers', 'student1', 'student2'],\n",
    "    index=oracle_names\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metrics_type</th>\n",
       "      <th>label</th>\n",
       "      <th>teacher</th>\n",
       "      <th>ensemblers</th>\n",
       "      <th>student1</th>\n",
       "      <th>student2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.196812</td>\n",
       "      <td>0.166924</td>\n",
       "      <td>0.157436</td>\n",
       "      <td>0.178076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.230684</td>\n",
       "      <td>0.195534</td>\n",
       "      <td>0.191666</td>\n",
       "      <td>0.191432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool_ml</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.196812</td>\n",
       "      <td>0.135528</td>\n",
       "      <td>0.157436</td>\n",
       "      <td>0.178076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy_ml</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.230684</td>\n",
       "      <td>0.214818</td>\n",
       "      <td>0.191666</td>\n",
       "      <td>0.191432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool_ml_x</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.196812</td>\n",
       "      <td>0.181630</td>\n",
       "      <td>0.157436</td>\n",
       "      <td>0.178076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.230684</td>\n",
       "      <td>0.178948</td>\n",
       "      <td>0.191666</td>\n",
       "      <td>0.191432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.123628</td>\n",
       "      <td>0.108272</td>\n",
       "      <td>0.133334</td>\n",
       "      <td>0.108418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.139714</td>\n",
       "      <td>0.128682</td>\n",
       "      <td>0.125338</td>\n",
       "      <td>0.112046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool_ml</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.123628</td>\n",
       "      <td>0.085832</td>\n",
       "      <td>0.133334</td>\n",
       "      <td>0.108418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy_ml</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.139714</td>\n",
       "      <td>0.154648</td>\n",
       "      <td>0.125338</td>\n",
       "      <td>0.112046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool_ml_x</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.123628</td>\n",
       "      <td>0.104846</td>\n",
       "      <td>0.133334</td>\n",
       "      <td>0.108418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.139714</td>\n",
       "      <td>0.103998</td>\n",
       "      <td>0.125338</td>\n",
       "      <td>0.112046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.521666</td>\n",
       "      <td>0.406666</td>\n",
       "      <td>0.231666</td>\n",
       "      <td>0.591666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.768334</td>\n",
       "      <td>0.450000</td>\n",
       "      <td>0.450000</td>\n",
       "      <td>0.766666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool_ml</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.521666</td>\n",
       "      <td>0.426666</td>\n",
       "      <td>0.231666</td>\n",
       "      <td>0.591666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy_ml</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.768334</td>\n",
       "      <td>0.525000</td>\n",
       "      <td>0.450000</td>\n",
       "      <td>0.766666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_bool_ml_x</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.521666</td>\n",
       "      <td>0.828334</td>\n",
       "      <td>0.231666</td>\n",
       "      <td>0.591666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp1_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp1</td>\n",
       "      <td>0.768334</td>\n",
       "      <td>0.733334</td>\n",
       "      <td>0.450000</td>\n",
       "      <td>0.766666</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             metrics_type  label   teacher  ensemblers  \\\n",
       "comp1_oracle_with_bool           f1_score  comp1  0.196812    0.166924   \n",
       "comp1_oracle_with_fuzzy          f1_score  comp1  0.230684    0.195534   \n",
       "comp1_oracle_with_bool_ml        f1_score  comp1  0.196812    0.135528   \n",
       "comp1_oracle_with_fuzzy_ml       f1_score  comp1  0.230684    0.214818   \n",
       "comp1_oracle_with_bool_ml_x      f1_score  comp1  0.196812    0.181630   \n",
       "comp1_oracle_with_fuzzy_ml_x     f1_score  comp1  0.230684    0.178948   \n",
       "comp1_oracle_with_bool          precision  comp1  0.123628    0.108272   \n",
       "comp1_oracle_with_fuzzy         precision  comp1  0.139714    0.128682   \n",
       "comp1_oracle_with_bool_ml       precision  comp1  0.123628    0.085832   \n",
       "comp1_oracle_with_fuzzy_ml      precision  comp1  0.139714    0.154648   \n",
       "comp1_oracle_with_bool_ml_x     precision  comp1  0.123628    0.104846   \n",
       "comp1_oracle_with_fuzzy_ml_x    precision  comp1  0.139714    0.103998   \n",
       "comp1_oracle_with_bool             recall  comp1  0.521666    0.406666   \n",
       "comp1_oracle_with_fuzzy            recall  comp1  0.768334    0.450000   \n",
       "comp1_oracle_with_bool_ml          recall  comp1  0.521666    0.426666   \n",
       "comp1_oracle_with_fuzzy_ml         recall  comp1  0.768334    0.525000   \n",
       "comp1_oracle_with_bool_ml_x        recall  comp1  0.521666    0.828334   \n",
       "comp1_oracle_with_fuzzy_ml_x       recall  comp1  0.768334    0.733334   \n",
       "\n",
       "                              student1  student2  \n",
       "comp1_oracle_with_bool        0.157436  0.178076  \n",
       "comp1_oracle_with_fuzzy       0.191666  0.191432  \n",
       "comp1_oracle_with_bool_ml     0.157436  0.178076  \n",
       "comp1_oracle_with_fuzzy_ml    0.191666  0.191432  \n",
       "comp1_oracle_with_bool_ml_x   0.157436  0.178076  \n",
       "comp1_oracle_with_fuzzy_ml_x  0.191666  0.191432  \n",
       "comp1_oracle_with_bool        0.133334  0.108418  \n",
       "comp1_oracle_with_fuzzy       0.125338  0.112046  \n",
       "comp1_oracle_with_bool_ml     0.133334  0.108418  \n",
       "comp1_oracle_with_fuzzy_ml    0.125338  0.112046  \n",
       "comp1_oracle_with_bool_ml_x   0.133334  0.108418  \n",
       "comp1_oracle_with_fuzzy_ml_x  0.125338  0.112046  \n",
       "comp1_oracle_with_bool        0.231666  0.591666  \n",
       "comp1_oracle_with_fuzzy       0.450000  0.766666  \n",
       "comp1_oracle_with_bool_ml     0.231666  0.591666  \n",
       "comp1_oracle_with_fuzzy_ml    0.450000  0.766666  \n",
       "comp1_oracle_with_bool_ml_x   0.231666  0.591666  \n",
       "comp1_oracle_with_fuzzy_ml_x  0.450000  0.766666  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_metrics[final_metrics.label=='comp1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metrics_type</th>\n",
       "      <th>label</th>\n",
       "      <th>teacher</th>\n",
       "      <th>ensemblers</th>\n",
       "      <th>student1</th>\n",
       "      <th>student2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.246490</td>\n",
       "      <td>0.187436</td>\n",
       "      <td>0.050794</td>\n",
       "      <td>0.258216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.243028</td>\n",
       "      <td>0.118800</td>\n",
       "      <td>0.112102</td>\n",
       "      <td>0.264080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool_ml</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.246490</td>\n",
       "      <td>0.290644</td>\n",
       "      <td>0.050794</td>\n",
       "      <td>0.258216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy_ml</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.243028</td>\n",
       "      <td>0.317284</td>\n",
       "      <td>0.112102</td>\n",
       "      <td>0.264080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool_ml_x</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.246490</td>\n",
       "      <td>0.277346</td>\n",
       "      <td>0.050794</td>\n",
       "      <td>0.258216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.243028</td>\n",
       "      <td>0.281350</td>\n",
       "      <td>0.112102</td>\n",
       "      <td>0.264080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.172528</td>\n",
       "      <td>0.154808</td>\n",
       "      <td>0.083332</td>\n",
       "      <td>0.194088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.164920</td>\n",
       "      <td>0.195556</td>\n",
       "      <td>0.150794</td>\n",
       "      <td>0.186646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool_ml</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.172528</td>\n",
       "      <td>0.197184</td>\n",
       "      <td>0.083332</td>\n",
       "      <td>0.194088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy_ml</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.164920</td>\n",
       "      <td>0.199784</td>\n",
       "      <td>0.150794</td>\n",
       "      <td>0.186646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool_ml_x</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.172528</td>\n",
       "      <td>0.163550</td>\n",
       "      <td>0.083332</td>\n",
       "      <td>0.194088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.164920</td>\n",
       "      <td>0.166646</td>\n",
       "      <td>0.150794</td>\n",
       "      <td>0.186646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.443290</td>\n",
       "      <td>0.253100</td>\n",
       "      <td>0.051516</td>\n",
       "      <td>0.418038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.470274</td>\n",
       "      <td>0.102310</td>\n",
       "      <td>0.102310</td>\n",
       "      <td>0.493794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool_ml</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.443290</td>\n",
       "      <td>0.620922</td>\n",
       "      <td>0.051516</td>\n",
       "      <td>0.418038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy_ml</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.470274</td>\n",
       "      <td>0.792064</td>\n",
       "      <td>0.102310</td>\n",
       "      <td>0.493794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_bool_ml_x</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.443290</td>\n",
       "      <td>0.933334</td>\n",
       "      <td>0.051516</td>\n",
       "      <td>0.418038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp2_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp2</td>\n",
       "      <td>0.470274</td>\n",
       "      <td>0.922222</td>\n",
       "      <td>0.102310</td>\n",
       "      <td>0.493794</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             metrics_type  label   teacher  ensemblers  \\\n",
       "comp2_oracle_with_bool           f1_score  comp2  0.246490    0.187436   \n",
       "comp2_oracle_with_fuzzy          f1_score  comp2  0.243028    0.118800   \n",
       "comp2_oracle_with_bool_ml        f1_score  comp2  0.246490    0.290644   \n",
       "comp2_oracle_with_fuzzy_ml       f1_score  comp2  0.243028    0.317284   \n",
       "comp2_oracle_with_bool_ml_x      f1_score  comp2  0.246490    0.277346   \n",
       "comp2_oracle_with_fuzzy_ml_x     f1_score  comp2  0.243028    0.281350   \n",
       "comp2_oracle_with_bool          precision  comp2  0.172528    0.154808   \n",
       "comp2_oracle_with_fuzzy         precision  comp2  0.164920    0.195556   \n",
       "comp2_oracle_with_bool_ml       precision  comp2  0.172528    0.197184   \n",
       "comp2_oracle_with_fuzzy_ml      precision  comp2  0.164920    0.199784   \n",
       "comp2_oracle_with_bool_ml_x     precision  comp2  0.172528    0.163550   \n",
       "comp2_oracle_with_fuzzy_ml_x    precision  comp2  0.164920    0.166646   \n",
       "comp2_oracle_with_bool             recall  comp2  0.443290    0.253100   \n",
       "comp2_oracle_with_fuzzy            recall  comp2  0.470274    0.102310   \n",
       "comp2_oracle_with_bool_ml          recall  comp2  0.443290    0.620922   \n",
       "comp2_oracle_with_fuzzy_ml         recall  comp2  0.470274    0.792064   \n",
       "comp2_oracle_with_bool_ml_x        recall  comp2  0.443290    0.933334   \n",
       "comp2_oracle_with_fuzzy_ml_x       recall  comp2  0.470274    0.922222   \n",
       "\n",
       "                              student1  student2  \n",
       "comp2_oracle_with_bool        0.050794  0.258216  \n",
       "comp2_oracle_with_fuzzy       0.112102  0.264080  \n",
       "comp2_oracle_with_bool_ml     0.050794  0.258216  \n",
       "comp2_oracle_with_fuzzy_ml    0.112102  0.264080  \n",
       "comp2_oracle_with_bool_ml_x   0.050794  0.258216  \n",
       "comp2_oracle_with_fuzzy_ml_x  0.112102  0.264080  \n",
       "comp2_oracle_with_bool        0.083332  0.194088  \n",
       "comp2_oracle_with_fuzzy       0.150794  0.186646  \n",
       "comp2_oracle_with_bool_ml     0.083332  0.194088  \n",
       "comp2_oracle_with_fuzzy_ml    0.150794  0.186646  \n",
       "comp2_oracle_with_bool_ml_x   0.083332  0.194088  \n",
       "comp2_oracle_with_fuzzy_ml_x  0.150794  0.186646  \n",
       "comp2_oracle_with_bool        0.051516  0.418038  \n",
       "comp2_oracle_with_fuzzy       0.102310  0.493794  \n",
       "comp2_oracle_with_bool_ml     0.051516  0.418038  \n",
       "comp2_oracle_with_fuzzy_ml    0.102310  0.493794  \n",
       "comp2_oracle_with_bool_ml_x   0.051516  0.418038  \n",
       "comp2_oracle_with_fuzzy_ml_x  0.102310  0.493794  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_metrics[final_metrics.label=='comp2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metrics_type</th>\n",
       "      <th>label</th>\n",
       "      <th>teacher</th>\n",
       "      <th>ensemblers</th>\n",
       "      <th>student1</th>\n",
       "      <th>student2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.296398</td>\n",
       "      <td>0.265862</td>\n",
       "      <td>0.253068</td>\n",
       "      <td>0.239716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.282690</td>\n",
       "      <td>0.253772</td>\n",
       "      <td>0.270234</td>\n",
       "      <td>0.221276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool_ml</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.296398</td>\n",
       "      <td>0.256130</td>\n",
       "      <td>0.253068</td>\n",
       "      <td>0.239716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy_ml</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.282690</td>\n",
       "      <td>0.252790</td>\n",
       "      <td>0.270234</td>\n",
       "      <td>0.221276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool_ml_x</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.296398</td>\n",
       "      <td>0.245874</td>\n",
       "      <td>0.253068</td>\n",
       "      <td>0.239716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>f1_score</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.282690</td>\n",
       "      <td>0.243598</td>\n",
       "      <td>0.270234</td>\n",
       "      <td>0.221276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.188550</td>\n",
       "      <td>0.164394</td>\n",
       "      <td>0.158658</td>\n",
       "      <td>0.144296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.170894</td>\n",
       "      <td>0.151054</td>\n",
       "      <td>0.160876</td>\n",
       "      <td>0.128074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool_ml</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.188550</td>\n",
       "      <td>0.152436</td>\n",
       "      <td>0.158658</td>\n",
       "      <td>0.144296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy_ml</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.170894</td>\n",
       "      <td>0.164916</td>\n",
       "      <td>0.160876</td>\n",
       "      <td>0.128074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool_ml_x</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.188550</td>\n",
       "      <td>0.141788</td>\n",
       "      <td>0.158658</td>\n",
       "      <td>0.144296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>precision</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.170894</td>\n",
       "      <td>0.140296</td>\n",
       "      <td>0.160876</td>\n",
       "      <td>0.128074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.754762</td>\n",
       "      <td>0.754762</td>\n",
       "      <td>0.697620</td>\n",
       "      <td>0.779762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.892858</td>\n",
       "      <td>0.871428</td>\n",
       "      <td>0.921428</td>\n",
       "      <td>0.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool_ml</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.754762</td>\n",
       "      <td>0.883334</td>\n",
       "      <td>0.697620</td>\n",
       "      <td>0.779762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy_ml</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.892858</td>\n",
       "      <td>0.860714</td>\n",
       "      <td>0.921428</td>\n",
       "      <td>0.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_bool_ml_x</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.754762</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.697620</td>\n",
       "      <td>0.779762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>comp4_oracle_with_fuzzy_ml_x</th>\n",
       "      <td>recall</td>\n",
       "      <td>comp4</td>\n",
       "      <td>0.892858</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.921428</td>\n",
       "      <td>0.900000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             metrics_type  label   teacher  ensemblers  \\\n",
       "comp4_oracle_with_bool           f1_score  comp4  0.296398    0.265862   \n",
       "comp4_oracle_with_fuzzy          f1_score  comp4  0.282690    0.253772   \n",
       "comp4_oracle_with_bool_ml        f1_score  comp4  0.296398    0.256130   \n",
       "comp4_oracle_with_fuzzy_ml       f1_score  comp4  0.282690    0.252790   \n",
       "comp4_oracle_with_bool_ml_x      f1_score  comp4  0.296398    0.245874   \n",
       "comp4_oracle_with_fuzzy_ml_x     f1_score  comp4  0.282690    0.243598   \n",
       "comp4_oracle_with_bool          precision  comp4  0.188550    0.164394   \n",
       "comp4_oracle_with_fuzzy         precision  comp4  0.170894    0.151054   \n",
       "comp4_oracle_with_bool_ml       precision  comp4  0.188550    0.152436   \n",
       "comp4_oracle_with_fuzzy_ml      precision  comp4  0.170894    0.164916   \n",
       "comp4_oracle_with_bool_ml_x     precision  comp4  0.188550    0.141788   \n",
       "comp4_oracle_with_fuzzy_ml_x    precision  comp4  0.170894    0.140296   \n",
       "comp4_oracle_with_bool             recall  comp4  0.754762    0.754762   \n",
       "comp4_oracle_with_fuzzy            recall  comp4  0.892858    0.871428   \n",
       "comp4_oracle_with_bool_ml          recall  comp4  0.754762    0.883334   \n",
       "comp4_oracle_with_fuzzy_ml         recall  comp4  0.892858    0.860714   \n",
       "comp4_oracle_with_bool_ml_x        recall  comp4  0.754762    1.000000   \n",
       "comp4_oracle_with_fuzzy_ml_x       recall  comp4  0.892858    1.000000   \n",
       "\n",
       "                              student1  student2  \n",
       "comp4_oracle_with_bool        0.253068  0.239716  \n",
       "comp4_oracle_with_fuzzy       0.270234  0.221276  \n",
       "comp4_oracle_with_bool_ml     0.253068  0.239716  \n",
       "comp4_oracle_with_fuzzy_ml    0.270234  0.221276  \n",
       "comp4_oracle_with_bool_ml_x   0.253068  0.239716  \n",
       "comp4_oracle_with_fuzzy_ml_x  0.270234  0.221276  \n",
       "comp4_oracle_with_bool        0.158658  0.144296  \n",
       "comp4_oracle_with_fuzzy       0.160876  0.128074  \n",
       "comp4_oracle_with_bool_ml     0.158658  0.144296  \n",
       "comp4_oracle_with_fuzzy_ml    0.160876  0.128074  \n",
       "comp4_oracle_with_bool_ml_x   0.158658  0.144296  \n",
       "comp4_oracle_with_fuzzy_ml_x  0.160876  0.128074  \n",
       "comp4_oracle_with_bool        0.697620  0.779762  \n",
       "comp4_oracle_with_fuzzy       0.921428  0.900000  \n",
       "comp4_oracle_with_bool_ml     0.697620  0.779762  \n",
       "comp4_oracle_with_fuzzy_ml    0.921428  0.900000  \n",
       "comp4_oracle_with_bool_ml_x   0.697620  0.779762  \n",
       "comp4_oracle_with_fuzzy_ml_x  0.921428  0.900000  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_metrics[final_metrics.label=='comp4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_metrics[final_metrics.label=='comp1'].to_csv('azure_comp1.csv')\n",
    "final_metrics[final_metrics.label=='comp2'].to_csv('azure_comp2.csv')\n",
    "final_metrics[final_metrics.label=='comp4'].to_csv('azure_comp4.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('.venv': poetry)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f70e5568322297c701f68d8cb002dc8dbcefa682adbbc7bc1b0f245279d2a68d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
